package com.terren.spider.run;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import org.apache.commons.collections.CollectionUtils;
import org.springframework.transaction.TransactionStatus;

import com.terren.spider.db.service.SpiderKeywordService;
import com.terren.spider.entity.common.SpiderLog;
import com.terren.spider.entity.core.Entry;
import com.terren.spider.entity.core.SpiderKeyword;
import com.terren.spider.run.basic.BasicSpiderRunner;
import com.terren.spider.util.common.BeanUtil;
import com.terren.spider.util.common.CommonUtil;
import com.terren.spider.util.common.StringUtil;


/**
 * 
 * 
 * 
 * 关键词任务runner 
 * 即 在表`t_spider_keyword`设置 任务，包含keyword关键字的entry都可以走这个流程 
 * ## 定时任务
 * 与SpiderKeywordTaskRunner的区别在于 SpiderKeywordRunner是固定的定时任务，而SpiderKeywordTaskRunner是临时任务，不需要重复的去执行
 * @author chengyu
 *
 */
public class SpiderKeywordRunner<T> extends BasicSpiderRunner<T> {
	protected Integer sourceId;
	protected Integer limit = 1;// 读取任务的条数
	protected SpiderKeywordService spiderKeywordService;
	protected List<SpiderKeyword> keywords;

	public SpiderKeywordRunner() {
		super();
		keywords = new ArrayList<>();
		spiderKeywordService = CommonUtil.getBean(SpiderKeywordService.class);
	}

	@Override
	public List<Entry> getEntrys() {
		List<Entry> entryList = new ArrayList<>();
		try {
			entryList = entryService.getEntryBySourceId(sourceId);
		} catch (Exception e) {
			log.error("SpiderUserRunner getEntrys error", e);
		}
		return entryList;
	}

	public void run() throws Exception {
		long startTime, endTime;
		startTime = System.currentTimeMillis();
		log.info("【爬虫开始】请耐心等待...");
		keywords = getKeywords();
		if (CollectionUtils.isNotEmpty(keywords)) {
			for (SpiderKeyword keywordTask : keywords) {
				// 记录爬虫日志
				this.sourceId = keywordTask.getSourceId();
				SpiderLog slog = new SpiderLog();
				List<Entry> entries = getEntrys();
				for (Entry entry : entries) {
					entry.setKeyword(keywordTask.getKeyword());
					String message = entry.getEntryDesc() + "爬取成功"+"[keyword:"+entry.getKeyword()+"]";
					Integer status = 1;// 默认成功
					Integer size = 0;
					try {
						postInitEntry(entry);
						// 记录爬虫日志
						slog = saveLog(entry);
						handler = spiderHandler(entry);// 根据catagory创建spider handler
						String url = formatUrl(entry, keywordTask);
						String[] targetUrls = handler.findUrl(url);
						size = handler.parsePage(targetUrls);
						

					} catch (Exception e) {
						message = entry.getEntryDesc() + "爬取失败"+"[keyword:"+entry.getKeyword()+"]" +e.getMessage();
						status = 0;
						throw e;
					} finally {
						// 更新爬虫日志
						updateLog(slog, entry, message, size, status);

					}
				}
				spiderKeywordService.updateFetchtimes(keywordTask);
				spiderKeywordService.updateDoneStatus(keywordTask);// status = 1
			}

		}

		endTime = System.currentTimeMillis();
		log.info("【爬虫结束】,共耗时约" + ((endTime - startTime) / 1000 / 60) + "分。");
	}

	public void runTask() {
		while (true) {
			try {
				run();
				Thread.sleep(5000);
			} catch (Exception e) {
				log.error("runTask error", e);
			}
		}
	}

	protected String[] formatUrls(Entry entry, List<SpiderKeyword> keywords) {
		Set<String> urls = new HashSet<>();
		for (SpiderKeyword task : keywords) {
			String id = task.getKeyword();
			String url = StringUtil.formatSearchUrl(id, entry.getEntryUrl());
			urls.add(url);
		}
	//	return urls.toArray(new String[urls.size()]);
		return BeanUtil.set2Array(urls);
	}

	protected String formatUrl(Entry entry, SpiderKeyword keyword) {
		String id = keyword.getKeyword();
		String url = StringUtil.formatSearchUrl(id, entry.getEntryUrl());
		return url;
	}

	protected List<SpiderKeyword> getKeywords() {
		TransactionStatus tran = null;// 事务
		List<SpiderKeyword> keywords = new ArrayList<>();
		try {
			tran = spiderKeywordService.startTransaction();// 开启事务
			SpiderKeyword task = new SpiderKeyword();
			//task.setSourceId(sourceId);
			task.setLimit(limit);
			keywords = spiderKeywordService.getKeywords(task);//
			if (CollectionUtils.isNotEmpty(keywords)) {
				spiderKeywordService.updateProgressStatus(keywords);// 任务正在进行//status = 9
			}

			spiderKeywordService.commitTransaction(tran);

		} catch (Exception e) {
			log.error("getKeywords error", e);
		} finally {
			spiderKeywordService.commitTransaction(tran);// 提交事务
		}
		return keywords;
	}
}
